# -*- coding: utf-8 -*-
#
# Copyright (c) 2011,2012 RMIT e-Research Office
#   (RMIT University, Australia)
# All rights reserved.
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#    *  Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#    *  Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#    *  Neither the name of RMIT University nor the
#       names of its contributors may be used to endorse or promote products
#       derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
"""
.. module:: QueryExpression
   :synopsis: Handles representation of a query string in a parseable form, which can 
then be turned into query strings etc.

.. moduleauthor:: Ian Thomas <Ian.Edward.Thomas@rmit.edu.au>

"""

import logging
from itertools import chain    
import ast


from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Q
from django.conf import settings

from smra.smra_portal.models import MediaObject 
from smra.smra_portal.models import MediaObjectParameter
from smra.smra_portal.models import MediaObjectParameterSet
from smra.smra_portal.models import Subquery
from smra.smra_portal.models import ParameterName

# Check for earlier version of booleano with incompatible API
try:
    from booleano.operations.converters import BaseConverter
except:
    from booleano.converters import BaseConverter
from booleano.parser import Grammar
from booleano.parser import ConvertibleParseManager
from booleano.exc import BooleanoException
    
logger = logging.getLogger(__name__)



# Special default (empty) namespace commands
DESCLABEL = "description"
MEDIATYPE = "mediatype"
IDLABEL   = "id"
SUBQUERYLABEL = "subquery" 

# special operators in our QueryExpression grammer.
CONTAINS_OP = ".contains."
EQUALS_OP = ".equals."

#FIXME: Change name to Conjunction
class QueryExpressionConverter(BaseConverter): 
    """ This is the interface for construction a parse tree in booleano.
        We override the unimplemented methods of BaseConverter to 
        create the structure we want, specifically BaseOp subclasses.
        We also register each type into lists so we can retrieve later. 
        
    """
    
    def __init__(self):
        BaseConverter.__init__(self)
        # Need to keep track of different ops created
        self.search_fields_ops = []
        self.ops = []
        
    def convert_not(self, operand):
        """ Create the unary negation"""
        uqo =  UnaryQueryOp(operand, " NOT ")
        # If we have negation, then indicate in child
        # SearchFieldOp that this has happened.
        operand.negate = True 
        self.ops.append(uqo)
        return uqo
     
    def convert_and(self, master_operand, slave_operand):
        """ Create the binary conjunction"""
        bqo = BinaryQueryOp(" AND ", master_operand, slave_operand)
        self.ops.append(bqo)
        return bqo
    
    def convert_or(self, master_operand, slave_operand):
        """ Create the binary disjunction"""
        bqo = BinaryQueryOp(" OR ", master_operand, slave_operand)
        self.ops.append(bqo)
        return bqo
        
    def convert_string(self, text):
        """ Create basic string.  Used for the value in the SearchFieldOp"""
        return (ParameterName.STRING, str(text))
    
    def convert_number(self, number):
        return (ParameterName.STRING, str(number))

    def convert_variable(self, name, namespace_parts):
        """ Create the variable.  Used for the field name in SearchFieldOp"""
        return (-1, name, namespace_parts) 
    
    def convert_equal(self, master_operand, slave_operand):
        """ Create the Searchfield term based on the
         LHS.contains.RHS operation"""
                 
        # work out type of RHS
        # TODO: refactor to be general
        if slave_operand[0] == ParameterName.STRING:
            slave_type = ParameterName.STRING
        elif slave_operand[0] < 0:
            # if find variable on RHS, assume it is a string
            slave_type = ParameterName.STRING
            
            
        master_type = master_operand[0]
        logger.debug("master_operand = %s" % str(master_operand))        
        logger.debug("master_type = %s" % master_type)
        logger.debug("slave type = %s" % slave_type)
        logger.debug("EQUALS_OP")
  
        
        if master_type < 0:
            # LHS is variable which is a search field
            search = SearchFieldOp(master_operand[2],
                                    slave_type,
                                    master_operand[1],
                                    EQUALS_OP,
                                    slave_operand[1]) 
            
        else:
            # LHS cannot be number or a string
            raise QueryExpressionError("LHS of equality must be a field type")            
        
        self.search_fields_ops.append(search)
        return search
    
    
    # This is the contains operator
    def convert_not_equal(self, master_operand, slave_operand):
        """ Create the Searchfield term based on the
         LHS.contains.RHS operation"""
                 
        # work out type of RHS
        # TODO: refactor to be general
        if slave_operand[0] == ParameterName.STRING:
            slave_type = ParameterName.STRING
        elif slave_operand[0] < 0:
            # if find variable on RHS, assume it is a string
            slave_type = ParameterName.STRING
            
            
        master_type = master_operand[0]
        logger.debug("master_operand = %s" % str(master_operand))        
        logger.debug("master_type = %s" % master_type)
        logger.debug("slave type = %s" % slave_type)
        logger.debug("CONTAINS_OP")
        
        if master_type < 0:
            # LHS is variable which is a search field
            search = SearchFieldOp(master_operand[2],
                                    slave_type,
                                    master_operand[1],
                                    CONTAINS_OP,
                                    slave_operand[1]) 
            
        else:
            # LHS cannot be number or a string
            raise QueryExpressionError("LHS of equality must be a field type")            
        
        self.search_fields_ops.append(search)
        return search
    
#    def convert_xor(self, master_operand, slave_operand):
#        raise NotImplementedError
#    
#    
#    def convert_less_than(self, master_operand, slave_operand):
#        raise NotImplementedError
#    
#    def convert_greater_than(self, master_operand, slave_operand):
#        raise NotImplementedError
#    
#    def convert_less_equal(self, master_operand, slave_operand):
#        raise NotImplementedError
#    
#    def convert_greater_equal(self, master_operand, slave_operand):
#        raise NotImplementedError
#    
#    def convert_belongs_to(self, master_operand, slave_operand):
#        raise NotImplementedError
#    
#    def convert_is_subset(self, master_operand, slave_operand):
#        raise NotImplementedError 
#    
#    def convert_number(self, number):
#        raise NotImplementedError
#    
#    def convert_set(self, *elements):
#        raise NotImplementedError
#    
#    def convert_function(self, name, namespace_parts, *arguments):
#        raise NotImplementedError
       
class BaseOp():
    """ The base for nodes in the Query Expression tree"""
    pass
    
    
class SearchFieldOp(BaseOp):
    """ A search value matched to e a search field in a specific schema.
    """
    
    def __init__(self, schema, field_type, field, operator, value):
        self.schema = schema # TODO: specify schemas to handle parameterSets
        self.field = field
        self.value = value
        self.negate = False
        self.type = field_type
        self.operator = operator
        
    def __repr__(self):
        """ The way a search field will look in the QueryExpression"""
        # We use .contains. rather than == because the operations is
        # a subset match rather than an exact match.  If later we do 
        # add exact match, then we will use == for that.
        if self.schema:
            return "%s:%s%s'%s'" % (self.schema, self.field,
                                    self.operator, self.value)
        else:
            return "%s%s'%s'" % (self.field, self.operator, str(self.value))

    
class UnaryQueryOp(BaseOp):
    """ A unary operator such as NOT """
    def __init__(self, first, opsym):
        self.first = first
        self.opsym = opsym
        
    def __repr__(self):
        return "%s%s" % (self.opsym, self.first)
    
      
class BinaryQueryOp(BaseOp):
    """ A binary operator such as AND, OR """
    
    def __init__(self, opsym, first, second):
        self.first = first
        self.second = second
        self.opsym = opsym # TODO: make enum
        
    def __repr__(self):
        return "%s%s%s" % (repr(self.first),
                           str(self.opsym),
                           repr(self.second))
    
    
class Error(Exception):
    """ The base exception for query expressions"""
    pass


class QueryExpressionError(Error):
    """ The query expression has invalid syntax"""
    pass


class QueryExpression():
    """ Holds a query expression, created from a search string"""
    
    def __init__(self, query=""):
        self.search_field_ops = []
        self.ops = []
        self.is_valid = False
        self.original_query = None
        self.expression_tree = None
        self.init_simple_search(query)
    
        
        
    def _make_expression_tree(self, bool_query_str):
        """ Parse boolean expression to produce expression_tree"""
        self.expression_tree = ''
        converter = QueryExpressionConverter()
        
        if bool_query_str:
            logger.debug("bool_query_str=%s" % bool_query_str)
            grammar = Grammar(eq=EQUALS_OP,ne=CONTAINS_OP,
                                   namespace_separator='/',
                                   identifier_spacing='_-')
            parse_manager = ConvertibleParseManager(grammar)
            
            logger.debug('bool_query= %s' % bool_query_str)
            self.expression_tree = ''
            try:
                parse_tree = parse_manager.parse(bool_query_str)
                logger.debug("parse_tree = %s" % parse_tree)
                self.expression_tree = parse_tree(converter)  
                logger.debug("conversion = %s" % self.expression_tree)
            except (BooleanoException, Exception), expr:
                self.expression_tree = 'ERROR'
                #traceback.print_exc(file=sys.stderr)
                raise QueryExpressionError(expr)
            self.search_field_ops = converter.search_fields_ops
            self.ops = converter.search_fields_ops 

    def init_simple_search(self, query=""):
        """ Creates a query expression from an query string"""     
        
        self.is_valid = False
        orig_query = query
        # First, build up boolean expression from query string manually         
        keywords = []
        while '"' in query:
            begin_quote = query.find('"')
            end_quote = query.find('"', begin_quote+1)
            quoted_keys = query[begin_quote:end_quote+1]
            if begin_quote > 0 and query[begin_quote-1] == '-':
                keywords.append("-" + quoted_keys.strip('"'))
                query = query.replace("-" + quoted_keys, ' ')
            else:
                keywords.append(quoted_keys.strip('"'))
                query = query.replace(quoted_keys, ' ')
        keywords.extend(query.split())
        # convert bare keywords into field lookups on equality
        # TODO: handle "description" parameter field and 
        # "description" object field confusion        
        bool_query = []
        for word in keywords:    
            if word[0] == '-': 
                bool_query.append('~%s%s"%s"' % (DESCLABEL,
                                                 CONTAINS_OP,
                                                 word[1:]))
            else:
                bool_query.append('%s%s"%s"'%(DESCLABEL, 
                                                  CONTAINS_OP,
                                                  word))
        
        # Now we have proper boolean expression, parse to create structure.
        self._make_expression_tree(' & '.join(bool_query))
        
        self.original_query = orig_query
        # If we get here then there have been no QueryExpression Errors called
        # so we can call this expression valid.
        self.is_valid = True

    def init_full_expression(self, expression):
        """ Creates an query expression from the advanced canonical form.
            Allows creation of multiple levels as input, but result 
            has only one level in this implementation."""
        self.is_valid = False
        # Assume input is already valid boolean expression and just parse
        self._make_expression_tree(expression)
        self.original_query = "VC:%s" % expression
        # If we get here then there have been no QueryExpression Errors called
        # so we can call this expression valid.
        self.is_valid = True
        
    @staticmethod
    def _get_objs_with_desc(objs, operator,value, neg):
        """ Return the new set of objects if the field is a description field"""
        if operator == CONTAINS_OP:
            if (neg):
                objs = objs.exclude(description__contains=value)
            else:
                objs = objs.filter(description__contains=value)
        elif operator == EQUALS_OP:
            if (neg):
                objs = objs.exclude(description__iexact=value)
            else:
                objs = objs.filter(description__iexact=value)
            
        return objs
    
    @staticmethod
    def _get_objs_with_mediatype_orig(objs, value, neg):
        """ Return the new set of objects if the field is a mediatype"""
      
    
    @staticmethod
    def _get_objs_with_mediatype(objs, value, neg):
        """ Return the new set of objects if the field is a mediatype"""
        if settings.ORIG_QUERY:
            paramset = MediaObjectParameterSet.objects.filter(
                                                  schema__name__iexact=value)
            matchobjs = [p.media_object.pk for p in paramset]                    
            if neg:
                objs = objs.exclude(pk__in=matchobjs)
            else:
                objs = objs.filter(pk__in=matchobjs)
            return objs
        else:
        
            matchobjs = MediaObjectParameterSet.objects.filter(
                    schema__name__iexact=value).values_list('media_object__pk',
                                                            flat=True)
            if neg:
                objs = objs.exclude(pk__in=matchobjs)
            else:
                objs = objs.filter(pk__in=matchobjs)
            return objs

    @staticmethod
    def _get_objs_with_unknown(objs):
        """ Return the new set of objects if the field is unrecognised"""
        objs = MediaObject.objects.none() 
        return objs
    

        
    def get_matching_media_objects(self, objs,virt_id):
        """ Returns a QuerySet of Media Objects that matches the
            fields in the QueryExpression.    Key is that such calls can 
            be chained together into more complicated queries."""
        logger.debug("qe=%s" % self)
             
        logger.debug("initial objs = %s" % objs)
        
        # we don't parse the self.ops so assume that all queries
        # are one-level conjuctions of possibly negated literals.
        
        # if no fields in search, then return no objs
        
        if len(self.search_field_ops) == 0:
            return MediaObject.objects.none()
        
        for schema, field_type, field, operator, value, neg in [(
                                                    x.schema, x.type,
                                                    x.field, x.operator, 
                                                    x.value, x.negate)
                             for x in self.search_field_ops]:
            logger.debug("schema=%s field=%s operator=%s"
                         " value=%s neg=%s type=%s" % 
                         (schema, field, operator, value, neg, field_type))
                        
            if not schema:
                                                     
                # If empty (default) namespace then we are dealing with special
                # type
                
                if field.lower() == DESCLABEL:
                    
                    objs = QueryExpression._get_objs_with_desc(objs, 
                                                               operator,
                                                               value, 
                                                               neg)
                elif field.lower() == MEDIATYPE:                    
                    objs = QueryExpression._get_objs_with_mediatype(objs, 
                                                                    value, 
                                                                    neg)
                                 
                elif field.lower() == IDLABEL:
           
                    id_val = 0
                    try:
                        id_val = int(value)
                    except ValueError:
                        raise QueryExpressionError("invalid key" )
              
                    try:
                        id_obj = MediaObject.objects.get(pk=id_val) 
                        if (neg):
                            objs = objs.exclude(pk=id_obj.pk)
                        else:
                            objs = objs.filter(pk=id_obj.pk)
                            
                    except MediaObject.DoesNotExist:
                        if (neg):
                            objs = MediaObject.objects.all()
                        else:
                            objs = MediaObject.objects.none()
                    
                    # TODO: handle the neg case
#                    0
#                    # FIXME: There are serious performance implications
#                    # of the following approach, as we will lose lazy evaluation 
#                    # here.  Need to carefully consider whether there is a 
#                    # better way.    
#                    logger.debug("objs = %s" % objs)                             
#                    objs_list = [x.pk for x in chain(id_objs, objs)]
#                    logger.debug("chain is %s" % objs_list)
#                    objs = MediaObject.objects.filter(pk__in=objs_list)
#                    logger.debug("new objs %s" % objs)
#                    
#                    logger.debug("matchobjs=%s" % matchobjs)
                    
                    logger.debug("id_obj = %s" % id_obj)    
                      
                   
    
                elif field.lower() == SUBQUERYLABEL and virt_id:
                    
                    # Allow only pk identifiers for time being.
                    subquery_id = None                    
                    try:
                        subquery_id = int(value)
                    except ValueError, e:
                            raise QueryExpressionError(e)
                    
                    try:
                        subquery = Subquery.objects.get(#owned_by__exact=profile,
                                                       part_of=virt_id,
                                                       pk=subquery_id)
                    except ObjectDoesNotExist,e:
                        raise QueryExpressionError(e)
                    
                    
                    logger.debug("subquery.query= %s" % subquery.query)
                    
                         
                    disj = Disjunction(parse_disjunction(subquery.query))   
                 
                    logger.debug("disjunction is %s" % str(disj))
                    
                    try:
                        objs = disj.get_objects_for_disj(virt_id)
                    except QueryExpressionError:
                        raise QueryExpressionError(e)
                       
                        
                else:
                    objs = QueryExpression._get_objs_with_unknown(objs)  
                     
            else:
                                
                # so that multi-word field names are parsed
                # as multiple variables and not as a single string, so 
                # we need all whitespace as _ in query string.  Now we 
                # change back to do the search proper
                field = field.replace("_"," ")
                if settings.ORIG_QUERY:
                        
                    logger.debug("replaced field = %s" % field)
                    logger.debug("schema[0]=%s" % schema[0])
                    
                    #type_keys = [ x for (x,y) in ParameterName.TYPES]
                    #type_conversion = dict([(x,x) for x in type_keys])
                    #type_conversion[ParameterName.YEAR] = ParameterName.NUMERIC
                    #type_conversion[ParameterName.DATE] = ParameterName.STRING
                    #logger.debug("type_conversion=%s" % type_conversion)
                    
                    # FIXME: DATE/YEAR have no equivalent RHS types (only have 
                    # to choose from STRING/NUMBER, therefore this type check
                    # will never match.  Can the value match be okay in almost
                    # all cases?
                    logger.debug("operator=%s" % operator)
                    if operator == EQUALS_OP:                    
                        param = MediaObjectParameter.objects.filter(
                                Q(paramset__schema__name__iexact=schema[0])                                                    
                                & Q(name__name__iexact=field)
                                & Q(value__iexact=value)
                               #     & Q(name__type=type_conversion[field_type])
                               #     & Q(name__type=field_type)
                                )
                    elif operator == CONTAINS_OP:
                        param = MediaObjectParameter.objects.filter(
                                Q(paramset__schema__name__iexact=schema[0])                                                    
                                & Q(name__name__iexact=field)
                                & Q(value__contains=value)
                               #     & Q(name__type=type_conversion[field_type])
                               #     & Q(name__type=field_type)
                                )
                    else:
                        raise QueryExpressionError("Invalid operator")
                                                   
                    logger.debug("param=%s" % param)
        
                    matchobjs = [p.paramset.media_object.pk for p in param]
                    
                    logger.debug("matchobjs=%s" % matchobjs)
                else:
                    logger.debug("replaced field = %s" % field)
                    logger.debug("schema[0]=%s" % schema[0])
                    matchobjs = MediaObjectParameter.objects.filter(
                                 Q(paramset__schema__name__iexact=schema[0])                                                    
                                & Q(name__name__iexact=field)
                                & Q(value__contains=value)
                                & Q(name__type=field_type)).values_list(
                                    'paramset__media_object__pk',flat=True)
                                                           
                if (neg):
                    objs = objs.exclude(pk__in=matchobjs)
                else:
                    objs = objs.filter(pk__in=matchobjs)
        return objs 
    
    def __repr__(self):
        return repr(self.expression_tree)
    
#    def _parse_query(self,query):
#        """ Parses the query and returns it if valid, else None."""
#        
#        isVC = query[0:3] == "VC:"            
#        if isVC:
#            query = query[3:]
#        else:
#            pass            
#        logger.debug("query=%s" % query)
#        if query:          
#            try:   
#                if isVC:
#                    self.init_full_expression(query) 
#                else:
#                    self.init_simple_search(query)
#            except QueryExpressionError, e:
#                logger.error("QueryExpressionError: error=%s" %  e)
#                return None
    
        def as_compact_form(self):
            """ Returns the query expression in a form suitable for storing 
            in history/user profile?. """
            # TODO: testing
            return self.expression_tree
   
   
def parse_disjunction(querystr):
    """ Parses a disjunction, which is a list of query expression strings
    (without VC prefix)"""
    # TODO: move into Disjuction
    # literal_eval is safe to use on untrusted data (python 2.6+)
    res = []
    logger.debug("querystr=%s" %querystr)
    try:
        qs = list(ast.literal_eval(querystr))
    except SyntaxError,e:
        raise QueryExpressionError(e)
    except ValueError,e:
        raise QueryExpressionError(e)
        
    logger.debug("qs = %s" % qs)
    for q in qs:
        qe = parse_query(q)
        logger.debug("qe = %s" % qe)
        res.append(qe)
       
    return res
    
def parse_query(querystr):
    """ Parses the simple or advanced query string and returns it if 
    valid, else None.
    """
    # TODO: move into QueryExpression
    isVC = querystr[0:3] == "VC:"            
    if isVC:
        query = querystr[3:]
    else:
        query = querystr
    logger.debug("query=%s" % query)
    qe = QueryExpression()
    logger.debug("qe=%s" % qe)
    if query:          
        # try:   
        if isVC:
            qe.init_full_expression(query) 
        else:
            qe.init_simple_search(query)
        # except QueryExpressionError, e:
        #     logger.error("QueryExpressionError: error=%s" %  e)
        #    return None
    logger.debug("qe=%s" % qe)
    
    return qe 
     

class Disjunction():
    """ The Union of a set of query expressions"""
    
    def __init__(self,query_expressions=[]):
        self.qes = query_expressions
        
    def __getitem__(self,key):
        return self.qes[key]
    
    def __repr__(self):
        
        return "%s" % list([x.original_query for x in self.qes])
        
    def __len__(self):
        return len(self.qes)
    
      
    def __iter__(self):
        for i in self.qes:
            yield i
         
                    
    def get_objects_for_disj(self,currentvirt):                                                  
        objs = MediaObject.objects.none()                      
        for qe in self.qes:           
            qe_objs = qe.get_matching_media_objects(
                                            objs=MediaObject.objects.all(),
                                            virt_id=currentvirt)       
            logger.debug("qe is %s" % str(qe))             
            # TODO: handle the neg case            
            # FIXME: There are serious performance implications
            # of the following approach, as we will lose lazy evaluation 
            # here.  Need to carefully consider whether there is a 
            # better way.    
            if logger.isEnabledFor(logger.debug):
                logger.debug("objs = %s" % objs)                             
            objs_list = (x.pk for x in chain(qe_objs, objs))
            
            if logger.isEnabledFor(logger.debug):
                logger.debug("chain is %s" % objs_list)
                
            objs = MediaObject.objects.filter(pk__in=objs_list)
                        
            if logger.isEnabledFor(logger.debug):
                logger.debug("new objs %s" % objs)               
        return objs 
                 
#    def get_matching_media_objects(self,profile,virt_id):
#        """ Returns a QuerySet of Media Objects that matches the
#            fields in the QueryExpression.    Key is that such calls can 
#            be chained together into more complicated queries."""
#        logger.debug("qe=%s" % self)
#        
#        objs = MediaObject.objects.none()
#                                        
#        for conjunction in self.qes:
#            qe = QueryExpression(query=" & ".join(conjunction))
#            qe_objs = qe.get_matching_media_objects(objs=MediaObject.objects.all(),profile,virt_id)
#            
#                    
#            # TODO: handle the neg case
#            
#            # FIXME: There are serious performance implications
#            # of the following approach, as we will lose lazy evaluation 
#            # here.  Need to carefully consider whether there is a 
#            # better way.    
#            logger.debug("objs = %s" % objs)                             
#            objs_list = [x.pk for x in chain(qe_objs, objs)]
#            logger.debug("chain is %s" % objs_list)
#            objs = MediaObject.objects.filter(pk__in=objs_list)
#            logger.debug("new objs %s" % objs)
#             
#        return objs
    
            
